import pandas as pd
import numpy as np
import matplotlib.pyplot as plt
import seaborn as sns
import os
import codecs
from sklearn.cross_validation import train_test_split


seed = 1024
np.random.seed(seed)

path = '../data/'


corpus = pd.read_pickle(path+'corpus.pkl')

#train and valid
y = corpus['label']
train,test,train_y,test_y=train_test_split(corpus,y,test_size=0.2,random_state=seed,stratify=y)

#valid and dev
valid,dev,valid_y,dev_y=train_test_split(test,test_y,test_size=0.5,random_state=seed,stratify=test_y)

#analysis
print('train samples percentange of corpus: {}%'.format(round((train.shape[0]/corpus.shape[0]) * 100,2)))
print('postive samples percentange of train: {}%'.format(round(train['label'].mean() * 100,2)))
print('postive samples percentange of valid: {}%'.format(round(valid['label'].mean() * 100,2)))
print('postive samples percentange of dev: {}%'.format(round(dev['label'].mean() * 100,2)))


pd.to_pickle(train,path+'train.pkl')
pd.to_pickle(valid,path+'valid.pkl')
pd.to_pickle(dev,path+'dev.pkl')


